#!/usr/bin/perl
#
# Copyright (c) 2020 NVI, Inc.
#
# This file is part of VLBI Field System
# (see http://github.com/nvi-inc/fs).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#

# 1.0 Initialize

require "getopts.pl";

&Getopts("ab:cdf:g:hlpr:st:uv");

if ($#ARGV < 0 &&!defined($opt_h) &&!defined($opt_v)) {
    print STDERR "Try: 'chk1024 -h'\n";
    exit -1;
}

if(defined($opt_v)) {
    print "[chk1024 1.3]\n";
    exit -1;
}

if (1<(defined($opt_b)+defined($opt_g)+defined($opt_t))) {
    print "options ";
    print "b " if defined($opt_b);
    print "g " if defined($opt_g);
    print "t " if defined($opt_t);
    print "are mutually exclusive\n";
    print STDERR "Try: 'chk1024 -h'\n";
    exit -1;
}
if (1<(defined($opt_b)+defined($opt_g)+defined($opt_r))) {
    print "options ";
    print "b " if defined($opt_b);
    print "g " if defined($opt_g);
    print "r " if defined($opt_r);
    print "are mutually exclusive\n";
    print STDERR "Try: 'chk1024 -h'\n";
    exit -1;
}

if (defined($opt_b) && $opt_b ne "a" && $opt_b ne "e" && $opt_b ne "f") {
    print "options -b argument must be a|e|f\n";
    print STDERR "Try: 'chk1024 -h'\n";
    exit -1;
}
if (defined($opt_g) && $opt_g ne "a" && $opt_g ne "e" && $opt_g ne "f") {
    print "options -g argument must be a|e|f\n";
    print STDERR "Try: 'chk1024 -h'\n";
    exit -1;
}

if (defined($opt_h)) {
    print "Usage: chk1024 -acdhlpsuv -b a|e|f -f frames -g a|e|f -r track -t track logs\n";
    print "Synopsis: analyzes results of Mark5 check schedule v/chk1024 logs\n\n";

    print "This script will summarize the results of v/chk1024 or other similar Mark5\n";
    print "test schedule. The modes are determined from presence of 'setupX' procedure\n";
    print "calls in the log. Where 'X' is a string. The mode is either 'play' or 'idle'\n";
    print "depending on whether or not 'recscan' has been called since the last\n";
    print "'setupX'.\n\n";

    print "By default, chk1024 prints only the overall summary results grouped by setup.\n";
    print "The number of tracks checked reflect those that were found in the log to have\n";
    print "been selected with the Mark5.\n\n";

    print "The checks for each track are:\n";
    print "  Aux:    considered good if decoded track number is correct\n";
    print "  Errors: considered good if synch, resynch, parity and CRC errors were zero\n";
    print "  Frames: considered good if more than the minimum frames were processed,\n";
    print "          minimum frames is settable, but defaults to 300 (0x12c)\n\n";

    print "Additional detail can be displayed by selecting command line options. The\n";
    print "next level of detail is selected by the -s option, which will print a table\n";
    print "of results for each check by setup and mode, also summarizing them by 'set'\n";
    print "and odd/even track number. The 'reported' tracks are those for which the\n";
    print "corresponding decoder output appears. If the 'Total' and 'reported' tracks\n";
    print "don't agree this usually means there were decoder communication errors. The\n";
    print "next level is shown with the -t option which shows the processed data for the\n";
    print "selected track. Finally, the -r option will print the raw log records for the\n";
    print "selected track.\n\n";

    print "Please note that the set (odd/even) statistics and summary results at the\n";
    print "end only report the total number of tracks found in the log, not the total\n";
    print "expected. The user must verify that total tracks are correct for each setup.\n";
    print "The 'Total Tracks' in the summary table is typically a good measure of the\n";
    print "expected number of tracks, but they are distributed over the sets.\n\n";

    print "If the scan_check fails for a setup the phrase 'ERROR: scan_check failed'\n";
    print "is included in the overall summary for the affected setup.\n\n";

    print "If the decoder firmware version is wrong or could not be determined a message\n";
    print "to that effect is printed.\n\n";

    print "If the same set-up occurs more than once, results from all but the last are\n";
    print "discarded. Note also that the 'Location' title line is printed once for each\n";
    print "time it is encountered in the log, with the corresponding log time stamp.\n";
    print "Typically, this happens for each time the schedule is started. Results from\n";
    print "previous runs are discarded only for set-ups with the same names. All\n";
    print "unique set-ups in the log are reported, even if they are from different runs.\n";
    print "For a given set-up and mode if a track or scan_check appears more than once,\n";
    print "the results for the last occurance is reported. A warning about 'Duplicate\n";
    print "DQA records' indicates either a track (or tracks) was checked more than once\n";
    print "or there was a decoder communication problem\n\n";

    print "Options explanation:\n";
    print " -a        ignore aux data when calculating performance\n";
    print " -b a|e|f  display  bad tracks for one of the Aux|Errors|Frames\n";
    print "           checks, cannot be used with -g, -t, or -r\n";
    print " -c        show raw scan_check data\n";
    print " -d        show raw decoder firmware version data\n";
    print " -f frames sets the minimum level of required frames, default is 300 (0x12c)\n";
    print " -g a|e|f  display good tracks for one if the Aux|Errors|Frames\n";
    print "           checks, cannot be used with -b, -t, or -r\n";
    print " -h        print this help information and stop\n";
    print " -l        lists each set-up as it is encountered, useful when a log contains\n";
    print "           more than one run, possibly on different dates\n";
    print " -p        'permissive', handles some Mark IV decoder old firmware\n";
    print "           communication errors\n";
    print " -r track  to display raw data for track, cannot be used with -b or -g\n";
    print " -s        print summary statistics\n";
    print " -t track  to display processed data for track, cannot be used with -b or -g\n";
    print " -u        print tracks with duplicate dqa records\n";
    print " -v        print program version information and stop\n";
    exit -1;
}

# 2.0 extract data

$location="UNKNOWN";

$bad_tracks=defined($opt_b)||defined($opt_g);
$bad_a=$opt_b eq "a";
$bad_e=$opt_b eq "e";
$bad_f=$opt_b eq "f";
$good_a=$opt_g eq "a";
$good_e=$opt_g eq "e";
$good_f=$opt_g eq "f";

if(!defined($opt_f)) {
    $frame_min=300;
} else {
    $frame_min=$opt_f;
}

if($opt_t) {
    print "Setup Mode Track      Frames     Parity      Nosync    Resync        CRC  Aux\n";
}
foreach $file (@ARGV) {
    if(!defined($save_file)) {
	$save_file=$file;
    }
    open(FILE,$file) || do {
	die "can't open $file: $!\n";
    };
    
    if($bad_tracks) {
	$first=1;
    }
#   print "file $file \n";
    while (<FILE>) {
	s/\cM$//; # handle bad ftp with control-Ms
	if (/\#mk5cn\#/||/\#matcn\#/) { #ignore echo output
	    next;
	}
	if(!defined($date)){
	    $date=substr($_,0,17);
	}
	if(/;location,([^,]*)/i) {
	    $location=$1;
	    $date=substr($_,0,17);
	    print "Location: $location, Date: $date, Log: $save_file\n";
	} elsif(/read 00006298/) {
	    $r6298=/read 00006298 00002035/;
	    print if defined($opt_d);
	} elsif(/read 0000629A/) {
	    $r629a=/read 0000629A 00002E32/;
	    print if defined($opt_d);
	} elsif(/[:;]setup(.*)/i) {
	    $setup=$1;
	    if(defined($opt_l)) {
		print "Set-up $setup encountered\n";
	    }
	    $mode='idle';
	    if(defined($config{$setup}{$mode})) {
		foreach $modes (keys %{$config{$setup}}) {
		    undef $config{$setup}{$modes};
		}
		undef $scan_check{$setup};
		foreach $track (keys %{$tracks{$setup}{$mode}}) {
		    undef $tracks{$setup}{$mode}{$track};
		    undef $aux{$setup}{$mode}{$track};
		    undef $frames{$setup}{$mode}{$track};
		    undef $errors{$setup}{$mode}{$track};
		    undef $count{$setup}{$mode}{$track};		    
		}
		print "Setup $setup seen again, discarding previous results.\n";
#		die "$setup already seen\n";
	    } else {
		$config{$setup}{$mode}=1;
	    }
	    $scan_check{$setup}=-1;
	    if($bad_tracks) {
		$new=1;
	    }
	} elsif(/recscan/i) {
	    $mode='play';
	    if($bad_tracks) {
		$new=1;
	    }
	    if(!defined($config{$setup}{$mode})) {
		$config{$setup}{$mode}=1;
	    }
	} elsif(/\!scan_check\?/i) {
	    print if defined($opt_c);
            ($front,@fields)=split('[ ]*[:;][ ]* ');
	    $scan_check{$setup}=0;
	    $scan_check{$setup}=1 if $fields[7] eq "0";
	} elsif(/\/scan_check\//i) {
	    print if defined($opt_c);
	    chop;
            (@fields)=split(',');
	    $scan_check{$setup}=0;
	    $scan_check{$setup}=1 if $fields[7] eq "0";
	} elsif(/track_set\? *0 *: *([^: ]+) *: *([^:; ]+) */i) {
	    $track1=$1;
	    $track2=$2;
	    $tracks{$setup}{$mode}{$track1}=1;
	    $tracks{$setup}{$mode}{$track2}=1;
	} elsif(/auxilliary_data/i) {
	    if(!defined($track1)||!defined($track2)) {
		print "'track-set?' output not found before first" .
		    " 'auxillary_data' record, giving up\n";
		exit(-1);
	    }
            ($front,@fields)=split(' ');
	    $aux1=substr($fields[2],0,2);
	    $aux2=substr($fields[6],0,2);
	    $t1 = $track1;
	    $t1 = $t1-60 if $t1 >= 100;
	    $t1 = sprintf "%02d",$t1;
	    $t2 = $track2;
	    $t2 = $t2-60 if $t2 >= 100;
	    $t2 = sprintf "%02d",$t2;
	    $aux{$setup}{$mode}{$track1}=$t1 eq $aux1;
	    $aux{$setup}{$mode}{$track2}=$t2 eq $aux2;
	    if($track1 == $opt_r) {
		print "$setup, $mode, tracks ($track1,$track2), raw: ";
		print ;
	    }
	    if($track2 == $opt_r) {
		print "$setup, $mode, tracks ($track1,$track2), raw: ";
		print ;
	    }
	    if($bad_a || $good_a) {
		if($good_a == $aux{$setup}{$mode}{$track1}) {
		    if($new) {
			if($first) {
			    $first=0;
			} else {
			    print "\n";
			}
			if($bad_a) {
			    print "Aux bad tracks $setup $mode: ";
			} else {
			    print "Aux good tracks $setup $mode: ";
			}
			$new=0;
		    }
		    print "$track1 ";
		}
		if($good_a == $aux{$setup}{$mode}{$track2}) {
		    if($new) {
			if($first) {
			    $first=0;
			} else {
			    print "\n";
			}
			if($bad_a) {
			    print "Aux bad tracks $setup $mode: ";
			} else {
			    print "Aux good tracks $setup $mode: ";
			}
			$new=0;
		    }
		    print "$track2 ";
		}
	    }
	} elsif(/decode4\/dqa/i || (defined($opt_p) && /decode4\/[d]?q[a]?/i)) {
	    if(!defined($track1)||!defined($track2)) {
		print "'track-set?' output not found before first" .
		    " 'dqa' decoder data, giving up\n";
		exit(-1);
	    }
            ($front,@fields)=split(' ');
	    $frames{$setup}{$mode}{$track1}=hex($fields[0])>=$frame_min;
	    $frames{$setup}{$mode}{$track2}=hex($fields[5])>=$frame_min;
	    $errors{$setup}{$mode}{$track1}=
		hex($fields[1])==0 && hex($fields[2])==0 &&
		hex($fields[3])==0 && hex($fields[4])==0;
	    $errors{$setup}{$mode}{$track2}=
		hex($fields[6])==0 && hex($fields[7])==0 &&
		hex($fields[8])==0 && hex($fields[9])==0;
	    $count{$setup}{$mode}{$track1}++;
	    $count{$setup}{$mode}{$track2}++;
	    if(defined($opt_u)) {
		if($count{$setup}{$mode}{$track1}>1
		   && $count{$setup}{$mode}{$track2}>1) {
		    print "duplicate dqa for tracks ($track1,$track2) in setup $setup mode $mode\n"; 
		} elsif($count{$setup}{$mode}{$track1}>1) {
		    print "duplicate dqa for track $track1 in setup $setup mode $mode\n"; 
		} elsif($count{$setup}{$mode}{$track2}>1) {
		    print "duplicate dqa for track $track2 in setup $setup mode $mode\n"; 
		}
	    }
	    if($track1 == $opt_r) {
		print "$setup, $mode, tracks ($track1,$track2), raw: ";
		print;
	    }
	    if($track2 == $opt_r) {
		print "$setup, $mode, tracks ($track1,$track2), raw: ";
		print;
	    }
	    if(defined($opt_t)) {
		if($track1 == $opt_t) {
		    $fr=hex($fields[0]);
		    $e1=hex($fields[1]);
		    $e2=hex($fields[2]);
		    $e3=hex($fields[3]);
		    $e4=hex($fields[4]);
		    printf " %4s %4s   %3s  %10d %10d %10d %10d %10d  %3s\n",
		    $setup,$mode,$opt_t,$fr,$e1,$e2,$e3,$e4,$aux1;
		}
		if($track2 == $opt_t) {
		    $fr=hex($fields[5]);
		    $e1=hex($fields[6]);
		    $e2=hex($fields[7]);
		    $e3=hex($fields[8]);
		    $e4=hex($fields[9]);
		    printf " %4s %4s   %3s  %10d %10d %10d %10d %10d  %3s\n",
		    $setup,$mode,$opt_t,$fr,$e1,$e2,$e3,$e4,$aux2;
		}
	    }
	    if($bad_e || $good_e || $bad_f || $good_f) {
		if(($good_e || $bad_e) &&
		   $good_e == $errors{$setup}{$mode}{$track1} ||
		   ($good_f || $bad_f) &&
		   $good_f == $frames{$setup}{$mode}{$track1}) {
		    if($new) {
			if($first) {
			    $first=0;
			} else {
			    print "\n";
			}
			if($bad_e) {
			    print "Error bad tracks $setup $mode: ";
			} elsif($good_e) {
			    print "Error good tracks $setup $mode: ";
			} elsif($bad_f) {
			    print "Fames bad tracks $setup $mode: ";
			} elsif($good_f) {
			    print "Frames good tracks $setup $mode: ";
			} else {
			    print "Unknown tracks list case $setup $mode: ";
			}
			$new=0;
		    }
		    print "$track1 ";
		}
		if(($good_e || $bad_e) &&
		   $good_e == $errors{$setup}{$mode}{$track2} ||
		   ($good_f || $bad_f) &&
		   $good_f == $frames{$setup}{$mode}{$track2}) {
		    if($new) {
			if($first) {
			    $first=0;
			} else {
			    print "\n";
			}
			if($bad_e) {
			    print "Error bad tracks $setup $mode: ";
			} elsif($good_e) {
			    print "Error good tracks $setup $mode: ";
			} elsif($bad_f) {
			    print "Fames bad tracks $setup $mode: ";
			} elsif($good_f) {
			    print "Frames good tracks $setup $mode: ";
			} else {
			    print "Unknown tracks list case $setup $mode: ";
			}
			$new=0;
		    }
		    print "$track2 ";
		}
	    }
#	    $fr=hex($fields[0]);
#	    $er=hex($fields[1]);
#	    print " $setup $mode $track1 $fields[0] $fields[1] $fr $er $track2 \n";
#	    print;
	}
    }
}


if($bad_tracks) {
    if(!$first) {
	print "\n"; 
    }
}

# 3.0 print out statistics

print "Location: $location, Date: $date, Log: $save_file\n" if $location eq "UNKNOWN";
if(defined($opt_s)) {
    print "Setup Mode Total  Frames Errors  Aux   Set1odd  Set1even  Set2odd  Set2even\n";
    print "           Tracks ----------------------bad/reported-----------------------\n";
}

foreach $setup (keys %config) {
    foreach $mode (keys %{$config{$setup}}) {
	$gooda=$bada=0;
	undef %tested_head_aux;
	undef %tested_head_frames;
	undef %tested_head_errors;
	undef %bad_head;
	if(defined($aux{$setup}{$mode})) {
	    foreach $track (keys %{$aux{$setup}{$mode}}) {
		$tested_aux{$setup}{$track}{$mode}=1;
		$tested_head_aux{int $track/100}{$track%2}{$track}=1;
		if(!$aux{$setup}{$mode}{$track}) {
		    $bada++;
		    if(!defined($opt_a)) {
			$bad{$setup}{$track}=1;
			$bad_head{int $track/100}{$track%2}{$track}=1;
		    }
		} else {
		    $gooda++;
		}
	    }
	}
	$totala=$gooda+$bada;
	
	$goodf=$badf=0;
	if(defined($frames{$setup}{$mode})) {
	    foreach $track (keys %{$frames{$setup}{$mode}}) {
		$tested_frames{$setup}{$track}{$mode}=1;
		$tested_head_frames{int $track/100}{$track%2}{$track}=1;
		if(!$frames{$setup}{$mode}{$track}) {
		    $badf++;
		    $bad{$setup}{$track}=1;
		    $bad_head{int $track/100}{$track%2}{$track}=1;
		} else {
		    $goodf++;
		}
	    }
	}
	$totalf=$goodf+$badf;
	
	$goode=$bade=0;
	if(defined($errors{$setup}{$mode})) {	
	    foreach $track (keys %{$errors{$setup}{$mode}}) {
		$tested_errors{$setup}{$track}{$mode}=1;
		$tested_head_errors{int $track/100}{$track%2}{$track}=1;
		if(!$errors{$setup}{$mode}{$track}) {
		    $bade++;
		    $bad{$setup}{$track}=1;
		    $bad_head{int $track/100}{$track%2}{$track}=1;
		} else {
		    $goode++;
		}
	    }
	}
	$totale=$goode+$bade;

	if(defined($count{$setup}{$mode})) {	
	    foreach $track (keys %{$count{$setup}{$mode}}) {
		if($count{$setup}{$mode}{$track} > 1) {
		    $dups=1;
		}
	    }
	}
	if($totala != scalar(keys %{$tracks{$setup}{$mode}})) {
	    $missing_aux=1;
	}

	if(defined($opt_s)) {
	    foreach $stack (0,1) {
		foreach $odd (0,1) {
		    $totalh[$stack][$odd]=0;
		    $badh[$stack][$odd]=0;
		    foreach $track (keys %{$tested_head_frames{$stack}{$odd}}) {
			if($tested_head_errors{$stack}{$odd}{$track} &&
			   (defined($opt_a) || $tested_head_aux{$stack}{$odd}{$track})) {
			       $totalh[$stack][$odd]++;
			       if($bad_head{$stack}{$odd}{$track}) {
				   $badh[$stack][$odd]++;
			       }
			   }
		    }
		}
	    }
	    printf "%4s %4s  %4d   %2d/%2d  %2d/%2d  %2d/%2d" .
		"   %2d/%2d     %2d/%2d    %2d/%2d     %2d/%2d\n",
		$setup,$mode,scalar(keys %{$tracks{$setup}{$mode}}),
		$badf,$totalf,$bade,$totale,$bada,$totala,
		$badh[0][1],$totalh[0][1],$badh[0][0],$totalh[0][0],
		$badh[1][1],$totalh[1][1],$badh[1][0],$totalh[1][0];
	}
    }
}
if(defined($opt_s)) {
    $hexd=sprintf "%1x",$frame_min;
    print "Frame check used a minimum of $frame_min (0x$hexd) frames/track.\n";
}

if(!%config) {
    print "No test setups were found.\n";
}
foreach $setup (keys %config) {
    $totalo=$bado=0;
    @tr_list=((keys %{$tested_aux{$setup}}),(keys %{$tested_errors{$setup}}),
	      (keys %{$tested_frames{$setup}}));
#
# find unique tracks
#    print "@tr_list\n";
    %seen= ();
    @tr_list= grep { ! $seen{$_} ++ } @tr_list;
#    print "@tr_list\n";
#
    foreach $track (@tr_list) {
	$fully_checked=1;
	foreach $mode (keys %{$config{$setup}}) {
	    $fully_checked &&= $tested_errors{$setup}{$track}{$mode} &&
		$tested_frames{$setup}{$track}{$mode} &&
		($opt_a || $tested_aux{$setup}{$track}{$mode});
	}
#	print "setup $setup $track check $fully_checked\n";
	if($fully_checked) {
	    $totalo++;
	    if($bad{$setup}{$track}) {
		$bado++;
	    }
	}
    }
    $goodo=$totalo-$bado;
    $totaltr=scalar(keys %{$tracks{$setup}{$mode}});
#    print "totaltr $totaltr totalo $totalo bado $bado\n";
    if($totaltr == 0) {
	$percent=0;
    } else {
	$percent=int 100.0*($goodo/$totaltr)+0.001;
    }
    print "Setup `$setup' tested $totaltr tracks, $goodo ";
    if($goodo == 1) {
	print "was good";
    } else {
	print "were good";
    }
    print " ($percent\%)";
    if($scan_check{$setup} == -1) {
	print ", ERROR: scan_check missing";
    } elsif($scan_check{$setup} == 0) {
	print ", ERROR: scan_check failed";
    }
    print ".\n";

}

if((defined($r6298) && !$r6298)||(defined($r629a) && !$r629a)) {
    print "Error: Decoder has wrong version number.\n";
} elsif ((defined($r6298) && $r6298 &&!defined($r629a))||
	 (defined($r629a) && $r629a &&!defined($r6298))) {
    print "Decoder firmware version only PARTIALLY detected, but correct that far.\n";    
} elsif (!defined($r6298)||!defined($r629a)) {
    print "Decoder firmware version number NOT detected.\n";
}
if(defined($dups)) {
    print "Duplicate DQA records exist for some tracks, bad AUX checks\?\n";
}
if(defined($missing_aux)) {
    print "Some AUX records were missing";
    if(!defined($opt_s)){
	print ", display summary with -s option";
    }
    print ".\n";
}
if(defined($opt_a)) {
    print "AUX data ignored (option -a) for 'Set' summaries and overall statistics.\n";
}
